package seotools.positions;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import java.io.*;
import java.net.SocketTimeoutException;
import java.net.URL;
import java.net.URLConnection;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * Date: 09.05.11
 */
public class Positions {
    public static final String SEP = ",";
//    public static final int DELAY = 1400;
//    public static final String PROJECT = "fgarden";
    public static final String PROJECT = "marketline";
//    public static final String PROJECT = "extramobi";
//    public static final String PROJECT = "extramobi-nokia";
//    public static final String PROJECT = "extramobi-se";
//    public static final String PROJECT = "extramobi-samsung";
//    public static final String PROJECT = "extramobi-se-accum";
//    public static final String PROJECT = "extramobi-samsung-accum";
    public static final List<String> exceptList = new ArrayList<String>();

//    public static String yandexCookie = "yandexuid=6114911320232940; fuid01=4d726ae12b1a1ac0.N9hJY3OOtWG-kEtar5NvfvMUs1Xo_8AUqbE-fG1uUYhwf2PZq1e_rMNyTIPcAliV5rOj5oRQaNMCW7Z2qn2HXaeA_Otjy40U2yeL3hFKkrbBAo6y88nb2eM2jOvZk-KQ; yp=1636457327.sp.nd:50:lang:!#1639122669.udn.c2hlbXBsaW5lcg==; my=YwECAAAnAgAALQECNgEBAA==; yabs-frequency=/3/zBK23ICB9m000HGm0oaO2uFBw04f60kE00MK0FyAV0000tS201Vq2jnI408Qxmey0001xN022kyAF0000G00/; aw=1_teJxilGdgNv2sKBBTu4TVHEgb1C1h5fyiKPBkxTJWBob/AAAAAP//AwCMvQoV#A#; L=W05VMUB1RmhXWwdMUndHBXprWnsFWHp6KwAjJCVvDQVBXFscOBUeJhMARg8hDSs3BhMKPUU/MAJFQE8aEyBgIg==.1323762669.9201.292293.a955bfb5a45f2f6543987920d706dca3; yandex_login=shempliner; Session_id=1323762669.12889.3.118550753.2:330838428:230.8:1323762669153:1842216327:13.80516.49027.bacf88ea28e1a0ca96776234113f2d30; t=w";
//    public static String yandexCookie = "yandexuid=6114911320232940; fuid01=4d726ae12b1a1ac0.N9hJY3OOtWG-kEtar5NvfvMUs1Xo_8AUqbE-fG1uUYhwf2PZq1e_rMNyTIPcAliV5rOj5oRQaNMCW7Z2qn2HXaeA_Otjy40U2yeL3hFKkrbBAo6y88nb2eM2jOvZk-KQ; yp=1636457327.sp.nd:50:lang:!#1646289553.udn.c2hlbXBsaW5lcg==; my=YwECAAAnAgAALQECAA==; yabs-frequency=/3/Tm805sWCN8yRFG1e3455hGO5L0oE/; L=RGlCFQMOfQBEe1h0A3EAX1pbZQsBVn5kIhInJxsCNhlZNAsaPiRnHTwDZlcCAQgAFSNSSGk5UUd9Ll84TjZbBA==.1330929553.9284.268535.5efa718a1e15b0076eac5832e50a5665; aw=1_UeJxi5GAAgf8AAAAA//8DAAFHAQkA#A#; spravka=dD0xMzMxMjg3MzM5O2k9OTEuMTIyLjEyOS40MTt1PTEzMzEyODczMzk5Nzg4MTI3MzM7aD1jZjg3NjE0Yzc3NjFmYjM2NGRhNGUwNzE0ZmIxMTY5MQ==";
    public static String yandexCookie = "\tyandexuid=9902611332089472; fuid01=4f5b452b2eeffb52.MV7NtO06WHM7QMjHwin48dFOkMVoxsNMMi0iQy6xNgYXTjtCP7Gp94e4OOLsnUEKBArwkXbzEIbF3VytHNP1KhZrwqYc4k5bNlWrHL4llNC9UxCXNQvvcKYBJv3wCrBO";
//    public static String googleCookie = "PREF=ID=c97245adb651c3f1:U=c0123334a2bc4fb2:LD=ru:NR=100:NW=1:TM=1310708899:LM=1313949801:SG=2:S=g__LYXZJ7DpCvpHv; NID=50=zmr06tfrq8D-yZYRWQXWyBUcbVint8lUuSGSLC2Tzs-jpUriZGNT5sT_-iGsR1xtl3krkT2Xg03q-4p-DzynP0nSUoSMiVieq3RectZsWlYdp2Fe15xWHoV-H7b6xV0D; SID=DQAAAL8AAABKbwUVtabOxULNbANM8qW8Z7787RkMDZYoY2Pbs4bI3U7Y--l3xnUle6qzpDMgq0E2pAhw80wTwv2Mh0YC4W0px7C9KJx4qPXIhH7Ht6wQgBQ628YarEXxhoIJNJeV0gdoIlSjJIK_p2-e_mkMstjTCXPcSgIDt3GSc85ibAJJcJB6tqFw43Ln6ItEgQBBdKwDUbmsVumLyHHcTuRkg-H4JFAo2QVV_1kX8eqk6ZdQ-1uka5-73x4eulM0Pmy2htc; HSID=AkMCqhsBI3Jl6wgFx";
    public static String googleCookie = "";

    static {
//        exceptList.add("htc");
//        exceptList.add("nokia");
//        exceptList.add("samsung");
//        exceptList.add("sony ericsson");
//        exceptList.add("iphone");
//        exceptList.add("accum");
//        exceptList.add("extramobi-bst-accum");
//        exceptList.add("extramobi-craftmann-accum");
//        exceptList.add("extramobi-motorola-accum");
//        exceptList.add("extramobi-motorola_rus-accum");
//        exceptList.add("extramobi-nokia-accum");
//        exceptList.add("extramobi-nokia-charge");
//        exceptList.add("extramobi-nokia_bat-accum");
//        exceptList.add("extramobi-nokia_bl-accum");
//        exceptList.add("extramobi-nokia_rus-accum");
//        exceptList.add("extramobi-nokia_zar-charge");
//        exceptList.add("extr amobi-samsung-accum");
//        exceptList.add("extramobi-samsung-charge");
//        exceptList.add("extramobi-samsung_rus-charge");
//        exceptList.add("extramobi-samsung_zar-charge");
//        exceptList.add("extramobi-se-accum");
//        exceptList.add("extramobi-se-charge");
//        exceptList.add("благо");
//        exceptList.add("вертозел");
//        exceptList.add("водоем");
//        exceptList.add("горка");
//        exceptList.add("дизсада");
//        exceptList.add("дизсадов");
//        exceptList.add("изгородь");
//        exceptList.add("ланддизайн");
//        exceptList.add("ландпроект");
//        exceptList.add("ландработы");
//        exceptList.add("мощение");
//        exceptList.add("озелуч");
//        exceptList.add("посадкадер");
//        exceptList.add("пруд");
//        exceptList.add("рулгазон");
//        exceptList.add("цветник");
    }

    private static final String DATE_FORMAT = "dd-MM-yyyy-HH-mm";
    public static int count = 0;

    public static void main(String[] args) throws IOException, InterruptedException {
        File projectDir = new File("projects\\" + PROJECT);
        positions(projectDir);
    }

    private static void positions(File projectDir) throws IOException, InterruptedException {
        if (isInExcept(projectDir.getName())) {
            System.out.println("Missed a dir '" + projectDir + "'");
            return;
        }
        File[] files = projectDir.listFiles();
        for (File file : files) {
            if (file.isDirectory()) {
                positions(file);
            } else {
                break;
            }
        }
        try {
            System.out.println("process a dir '" + projectDir.getAbsolutePath() + "'..");
            String phrasesPath = projectDir + "\\phrases.txt";
            String urlsPath = projectDir + "\\URLs";
            String resultsPath = projectDir + "\\";
            List<String> phrasesList = processPhrases(loadPhrases(phrasesPath));
            List<String> urlsList = loadURLs(urlsPath);

            long start = System.currentTimeMillis();
            Map<String, Map<String, Integer>> phraseYandexUrlPositions = getYandexPhraseURLPositions(phrasesList, urlsList);
            Map<String, Map<String, Integer>> phraseGoogleUrlPositions = getGooglePhraseURLPositions(phrasesList, urlsList);
            long end = System.currentTimeMillis();
            System.out.println("time = " + (end - start) / 60000 + " min.");
            StringBuilder sb = new StringBuilder();
            for (String phrase : phraseYandexUrlPositions.keySet()) {
                Map<String, Integer> yandexUrlPositions = phraseYandexUrlPositions.get(phrase);
                Map<String, Integer> googleUrlPositions = phraseGoogleUrlPositions.get(phrase);
                phrase = processPhraseBack(phrase);
                sb.append(phrase).append(SEP);
                System.out.print(phrase + ",");
                for (String url : yandexUrlPositions.keySet()) {
                    int yandexPosition = yandexUrlPositions.get(url);
                    int googlePosition = googleUrlPositions.get(url);
                    sb.append(url).append(SEP).append(yandexPosition).append(SEP);
                    sb.append(googlePosition).append(SEP);
                    System.out.print(url + "," + yandexPosition);
                    System.out.print("," + googlePosition + ",");
                }
                sb.append("\n");
                System.out.println();
            }
            saveResults(sb, resultsPath);
            System.out.println("finished");
        } catch (FileNotFoundException e) {
            System.out.println(e.getMessage());
        }
    }

    public static boolean isInExcept(String dirName) {
        return exceptList.contains(dirName);
    }

    public static void saveResults(StringBuilder sb, String path) throws IOException {
        FileOutputStream out = null;
        try {
            String date = new SimpleDateFormat(DATE_FORMAT).format(new Date()) + ".txt";
            out = new FileOutputStream(path + date);
            out.write(sb.toString().getBytes());
            out.flush();
        } finally {
            if (out != null) {
                out.close();
            }
        }
    }

    private static Map<String, Map<String, Integer>> getGooglePhraseURLPositions(List<String> phrasesList, List<String> urlsList) throws IOException, InterruptedException {
        Map<String, Map<String, Integer>> phraseUrlPositions = new LinkedHashMap<String, Map<String, Integer>>();
        for (String phrase : phrasesList) {
            Map<Integer, String> googlePositions = requestGooglePositions(phrase);

            Map<String, Integer> urlPositions = new HashMap<String, Integer>();
            // init urlPositions
            for (String url : urlsList) {
                urlPositions.put(url, 101);
            }
            for (Integer position : googlePositions.keySet()) {
                String site = googlePositions.get(position).toLowerCase();
                for (String url : urlsList) {
                    if (site.contains(url.toLowerCase())) {
                        urlPositions.put(url, position);
                    }
                }
            }
            phraseUrlPositions.put(phrase, urlPositions);
            googleDelay();
        }
        return phraseUrlPositions;
    }

    public static Map<String, Map<String, Integer>> getYandexPhraseURLPositions(List<String> phrasesList, List<String> urlsList) throws IOException, InterruptedException {
        Map<String, Map<String, Integer>> phraseUrlPositions = new LinkedHashMap<String, Map<String, Integer>>();
        for (String phrase : phrasesList) {
            Map<Integer, String> yandexPositions = requestYandexPositions(phrase);

            Map<String, Integer> urlPositions = new HashMap<String, Integer>();
            // init urlPositions
            for (String url : urlsList) {
                urlPositions.put(url, 101);
            }
            for (Integer position : yandexPositions.keySet()) {
                String site = yandexPositions.get(position).toLowerCase();
                for (String url : urlsList) {
                    if (site.contains(url.toLowerCase())) {
                        urlPositions.put(url, position);
                    }
                }
            }
            phraseUrlPositions.put(phrase, urlPositions);
        }
        return phraseUrlPositions;
    }

    public static Map<Integer, String> requestGooglePositions(String phrase) throws IOException, InterruptedException {
        Map<Integer, String> positions = new HashMap<Integer, String>();
        for (int page = 0; page < 100; page += 100) {
            if (page != 0) {
                googleDelay();
            }
            Document doc;
//            String url = "http://www.google.com/search?sclient=psy&hl=ru&source=hp&q=" + phrase + "&btnG=search&start=" + page;
            String googleURL = "http://www.google.ru/search?hl=ru&source=hp&biw=1280&bih=566&q="+phrase+"&aq=1&aqi=g10&aql=&oq=&start=" + page;
            System.out.println(googleURL);
            try {
                doc = Jsoup.connect(googleURL).
                        header("user-agent", "Mozilla/5.0 (Windows NT 5.1; rv:5.0) Gecko/20100101 Firefox/5.0").
                        header("Cookie", googleCookie).
                        timeout(20000).get();
            } catch (IOException e) {
                System.out.println(e + ": " + googleURL);
                Thread.sleep(500);
                page -= 100;
                continue;
            }
            Element res = doc.getElementById("rso");
            Elements cites = res.select("cite");
            int exceptions = 0;
            int position = 0;
            for (Element cite : cites) {
                position++;
                if (isException(cite.text())) {
                    exceptions++;
                }
                positions.put(position - exceptions + page, cite.text());
            }
        }
        return positions;
    }

    private static boolean isException(String text) {
        return text.contains("youtube.com") || text.contains("smotri.com");
    }

    public static Map<Integer, String> requestYandexPositions(String phrase) throws IOException, InterruptedException {
        return requestYandexPositions(phrase, 2, 50);
    }
    public static Map<Integer, String> requestYandexPositions(String phrase, int region, int pages) throws IOException, InterruptedException {
        Map<Integer, String> positions = new HashMap<Integer, String>();
        int maxPage = pages > 50 ? 2 : 1;
        for (int page = 0; page < maxPage; page++) {
            if (page != 0) {
                yandexDelay();
            }
            Document doc;
            String url = "http://yandex.ru/yandsearch?p=" + page + "&text=" + phrase + "&lr=" + region + "&numdoc=" + pages;
            System.out.println(url);
            try {
                doc = Jsoup.connect(url).
                        header("user-agent", "Mozilla/5.0 (Windows NT 5.1; rv:2.0.1) Gecko/20100101 Firefox/4.0.1").
                        header("Cookie", yandexCookie).
                        timeout(20000).get();
            } catch (IOException e) {
                processCaptcha(e);
                page--;
                continue;
            }
            Element itemList = doc.select("ol.b-serp-list").get(0);
            Elements items = itemList.select(".b-serp-item__title");
            for (Element item : items) {
                if (item.select("b.b-serp-item__number").size() == 0) {
                    continue;
                }
                String href = item.child(1).attr("href");
                if (href.contains("/infected?url=")) {
                    continue;
                }
                Integer position = Integer.parseInt(item.child(0).text());
                positions.put(position, href);
            }
        }
        return positions;
    }

    private static void processCaptcha(IOException e) throws IOException, InterruptedException {
        try {
            Pattern p = Pattern.compile("http://[a-zA-Z0-9%/?&.=]+");
            Matcher m = p.matcher(e.getMessage());
            m.find();
            System.out.println("GOT CAPTCHA!!!");
//            String captchaURL = e.getMessage().substring(m.start(), m.end() - 1);
//            URLConnection conn = new URL(captchaURL).openConnection();
//            conn.connect();
//            Document doc = Jsoup.parse(conn.getInputStream(), "UTF-8", "yandex.ru");
//            String captchImg = doc.select("img.b-captcha__image").get(0).attr("src");
//            String key = doc.select("form input[name=key").get(0).text();
//            String retpath = doc.select("form input[name=retpath").get(0).text();
            InputStreamReader in = new InputStreamReader(System.in);
//            System.out.println(captchImg);
            String newCookie = new BufferedReader(in).readLine();
            yandexCookie = newCookie;
//            conn =  new URL(captchaURL).openConnection();
//            conn.setDoOutput(true);
//            PrintStream ps = new PrintStream(conn.getOutputStream());
//            ps.println("http://yandex.ru/checkcaptcha?key=" + key + "&retpath=" + retpath + "&rep=" + rep);
//            ps.flush();
//            ps.close();
            yandexDelay();
        } catch (Exception e1) {
            e.printStackTrace();
        }
    }

    public static void yandexDelay() throws InterruptedException {
        count++;
        int delay = 3000 + (int) (Math.random() * 2500);
//        System.out.println(adCount + ". " + delay);
        Thread.sleep(delay);
    }

    public static void googleDelay() throws InterruptedException {
        count++;
        int delay = 1500 + (int) (Math.random() * 1200);
//        System.out.println(adCount + ". " + delay);
        Thread.sleep(delay);
    }
//    public static Map<Integer, String> requestYandexPositions(String phrase) throws IOException, InterruptedException {
//        Map<Integer, String> positions = new HashMap<Integer, String>();
//        for (int page = 0; page < 10; page++) {
//            if (page != 0) {
//                Thread.sleep(DELAY);
//            }
//            Document doc = null;
//            String url = "http://yandex.ru/yandsearch?p=" + page + "&text=" + phrase + "&lr=2";
//            try {
//                doc = Jsoup.connect(url).
//                        header("user-agent", "Mozilla/5.0 (Windows NT 5.1; rv:2.0.1) Gecko/20100101 Firefox/4.0.1").
////                        header("Cookie", "spravka=dD0xMzA2MzA1MTA2O2k9MTA5LjIwNS4yNTMuMTM1O3U9MTMwNjMwNTEwNjMxNDIzNDAwMTtoPTk3MzNiNTBiNjU0ZGQ4MDVlY2NiYzZkYzAzOGQzNzQz; yandexuid=6755421306305106; fuid01=4d726ae12b1a1ac0.N9hJY3OOtWG-kEtar5NvfvMUs1Xo_8AUqbE-fG1uUYhwf2PZq1e_rMNyTIPcAliV5rOj5oRQaNMCW7Z2qn2HXaeA_Otjy40U2yeL3hFKkrbBAo6y88nb2eM2jOvZk-KQ; t=p; yabs-frequency=/3/Tm805_G7N55001Vq1q40/").get();
//        header("Cookie", "yandexuid=5755361306351250; fuid01=4d726ae12b1a1ac0.N9hJY3OOtWG-kEtar5NvfvMUs1Xo_8AUqbE-fG1uUYhwf2PZq1e_rMNyTIPcAliV5rOj5oRQaNMCW7Z2qn2HXaeA_Otjy40U2yeL3hFKkrbBAo6y88nb2eM2jOvZk-KQ").
//                timeout(60000).get();
//            } catch (IOException e) {
//                System.out.println(e + ": " + url);
//                Thread.sleep(500);
//                page--;
//                continue;
//            }
//            Element itemList = doc.select("ol.b-serp-list").get(0);
//            Elements items = itemList.select(".b-serp-item__title");
//            for (Element item : items) {
//                Integer position = Integer.parseInt(item.child(0).text());
//                positions.put(position, item.child(1).attr("href"));
//            }
//        }
//        return positions;
//    }

    public static List<String> loadPhrases(String path) throws IOException {
        BufferedReader reader = null;
        try {
            reader = new BufferedReader(new InputStreamReader(new FileInputStream(path)));
            List<String> phrases = new ArrayList<String>();
            String phrase;
            while ((phrase = reader.readLine()) != null) {
                phrases.add(phrase);
            }
            return phrases;
        } finally {
            if (reader != null) {
                reader.close();
            }
        }
    }

    public static List<String> loadURLs(String path) throws IOException {
        BufferedReader reader = null;
        try {
            reader = new BufferedReader(new InputStreamReader(new FileInputStream(path)));
            List<String> urls = new ArrayList<String>();
            String url;
            while ((url = reader.readLine()) != null) {
                urls.add(url);
            }
            return urls;
        } finally {
            if (reader != null) {
                reader.close();
            }
        }
    }

    public static List<String> processPhrases(List<String> phrases) {
        List<String> newPhrases = new ArrayList<String>(phrases.size());
        for (String phrase : phrases) {
            if (phrase.charAt(0) == 65279) {
                phrase = phrase.substring(1);
            }
            phrase = phrase.replaceAll(" ", "+");
            newPhrases.add(phrase);
        }
        return newPhrases;
    }

    public static String processPhraseBack(String phrase) throws UnsupportedEncodingException {
//        phrase = new String(phrase.getBytes("UTF-8"), "windows-1251");
        return phrase.replaceAll("\\+", " ");
    }
}
